import argparse
import json
import logging
import os
from pathlib import Path
import platform
import re
import shutil
import subprocess
import sys
import tarfile
import time
import traceback
from zipfile import ZipFile
import zipfile
import tempfile


ESP_ADDRESS_SPACE_BOOT_ESP32   = 0x1000
ESP_ADDRESS_SPACE_BOOT_ESP32P4 = 0x2000
ESP_ADDRESS_SPACE_BOOT_ESP32S3 = 0x0000

ESP_ADDRESS_SPACE_PART_TABLE   = 0x8000
ESP_ADDRESS_SPACE_PART_TABLE_P4 = 0x9000

ESP_ADDRESS_SPACE_MAIN         = 0x10000

SUPPORTED_CHIP_TYPES = ["esp32", "esp32s3", "esp32p4"]

MD_FILENAME = "md.json"
MD_BUILD_FILENAME = "esp_builder_md.json"
BOOTLOADER_FILENAME="bootloader.bin"
PARTTABLE_FILENAME="partition-table.bin"


def executeExternalSubprocess(arg_list, path, file2Output=None, env_list = None, full_output = False, err_file = None, xshell=False, no_output=False, no_copy_env=False):
    ret_code = -1000
    output_descr = subprocess.PIPE
    output_text = ""
    try:
        #if(not os.path.isfile(arg_list[0])):
        #    return -1, "Unknown file"
        if(file2Output != None):
            output_descr = open(file2Output, "w")
        if no_copy_env:
            env = env_list
        else:
            env = os.environ.copy() 
            if(env_list != None):
                for env_entry in env_list:
                    if len(env_entry) > 2:
                        if(env_entry[1] == "+"):
                            env[env_entry[0]] += env_entry[2]
                    else:
                        env[env_entry[0]] = env_entry[1]

        #if(DUMP_ARGS):      
        #print( "\n\tSubprocess args:%s\n"%(arg_list))

        #proc = subprocess.Popen(arg_list, cwd=path, stdout=output_descr, stderr = subprocess.PIPE, env = env)
        # subprocess.PIPE
        if(full_output):
            proc = subprocess.Popen(arg_list, cwd=path, env = env, shell = xshell, stderr = sys.stdout)
        else:
            err_descr = output_descr
            if(err_file != None):
                err_descr = open(err_file, "w")
            proc = subprocess.Popen(arg_list, cwd=path, stdout=output_descr, stderr = err_descr, env = env, shell = xshell)
            
        proc.wait()
        
        if(file2Output != None and (not no_output)):
            output_descr.close()
            with open(file2Output, "r") as f:
                output_text = f.read()
        ret_code = proc.returncode
        #logging.debug(" >>> %s\nreturns: %s\nworking dir: %s" % (' '.join(arg_list), ret_code, path))

    except Exception as e1:
        logging.error("Exception:%s"%(e1))
        traceback.print_exc()
        ret_code -2, "%s"%(e1)
    return ret_code, output_text
def calcDirectorySize(start_path = '.'):
    total_size = 0
    for dirpath, dirnames, filenames in os.walk(start_path):
        for f in filenames:
            fp = os.path.join(dirpath, f)
            # skip if it is symbolic link
            if not os.path.islink(fp):
                total_size += os.path.getsize(fp)
    return total_size

def name2ArchMode(filename):
    if filename.endswith(".tar.xz"):
        return ":xz"
    if filename.endswith(".tar.gz"):
        return ":gz"
    if filename.endswith(".tar.bz2"):
        return ":bz2"
    return ":gz"
my_tar_mode =0o777
def tarModeFilter(tarinfo):
    global my_tar_mode
    tarinfo.mode = my_tar_mode
    tarinfo.uid = tarinfo.gid = 0
    tarinfo.uname = tarinfo.gname = "root"
    return tarinfo
def packArchive(src_folder, dstname, filter=None, progress_cbk = None, fix_acess_map = 0o777, common_prefix = ""):
    total_size = 0
    sum_size = 0
    global my_tar_mode
    if progress_cbk != None:
        total_size = calcDirectorySize(src_folder)
    zip_mode = False
    if dstname.endswith(".zip"):
        zipObj = ZipFile(dstname, 'w', zipfile.ZIP_DEFLATED)
        zip_mode = True
        #zipObj.open(dstname, 'w')
    else:
        compression_mode = name2ArchMode(dstname)
        zipObj = tarfile.open(dstname, "x"+compression_mode)
        if fix_acess_map != None:
            my_tar_mode = fix_acess_map
        #zipObj.open(dstname, "x"+compression_mode)

    # Iterate over all the files in directory
    for folderName, subfolders, filenames in os.walk(src_folder):
        if folderName.find(".DS_Store") != -1:
            continue
        for filename in filenames:
            #create complete filepath of file in directory
            filePath = os.path.join(folderName, filename)
            # Add file to zip
            arch_path = filePath.replace(src_folder, "")
            if common_prefix != None:
                #print("COMMON PREFIX:%s"%(common_prefix))
                arch_path = common_prefix + arch_path #os.path.join(common_prefix, arch_path)
            if zip_mode:
                with open(filePath, "rb") as f:
                    bytes = f.read()
                #print("arch_path:%s"%(arch_path))
                info = zipfile.ZipInfo.from_file(filePath, arch_path)
                if fix_acess_map != None:
                    info.external_attr |= fix_acess_map << 16  # give full access to included file
                    info.compress_type = zipObj.compression
                #print("ZIP")
                zipObj.writestr(info, bytes)
            else:
                if fix_acess_map != None:
                    zipObj.add(filePath, arch_path, filter=tarModeFilter)
                else:
                    zipObj.add(filePath, arch_path)
            if progress_cbk != None:
                sum_size += os.path.getsize(filePath)
                progress_cbk(sum_size, total_size)
    zipObj.close()
def loadSourceFile(filename, by_lines = True):
    logging.debug("Loading source file:%s"%(filename))
    source_lines = []
    with open(filename, "r", encoding='utf-8', errors='ignore') as f:
        if(by_lines):
            source_lines = f.readlines()
        else:
            source_lines = f.read()
    return source_lines
def loadBinaryFile(filename):
    logging.debug("Loading binary file:%s"%(filename))
    source_lines = []
    with open(filename, "rb") as f:
        source_data = f.read()
    return source_data
def saveTextFile(text_filename, string_data):
    with open(text_filename, 'w', encoding='utf-8', errors='ignore') as the_file:
        the_file.write(string_data)
def loadJSONData(filename):
    if os.path.isfile(filename):
        ts = time.time()
        text = loadSourceFile(filename, False)
        #md = orjson.loads(text)
        md = json.loads(text)
        logging.info("*** JSON %s load elapsed:%3.3f"%(filename, time.time() - ts))
        return md
    return None 
def dumpJSONData(filename, metadata, sort=True):
    try:
        #text = orjson.dumps(metadata).decode('utf-8')
        text = json.dumps(metadata)
        saveTextFile(filename, text)
    except Exception as e1:
        logging.error("Exception:%s"%(e1))
        traceback.print_exc()
def zmeUserStoragePath(sub_path = None):
    path = os.path.expanduser("~")+os.sep+"ZMEStorage"+ os.sep
    if sub_path != None:
        if not sub_path.endswith(os.sep):
            sub_path += os.sep
        path += sub_path
    if not os.path.isdir(path):
        os.makedirs(path)
    return path
def zmeSetupLogging(name, bVerbose=True, bNoStdOutput=False):
    fn =  '%s/%s-%s.log' % (zmeUserStoragePath(), name, time.strftime("%Y-%m-%d", time.gmtime()))
    format_str = '%(levelname)-8s [%(asctime)s]  %(message)s'
    lev = logging.WARNING
    if bVerbose:
        lev = logging.DEBUG
    logging.basicConfig(format=format_str, level=lev, filename=fn)
    if not bNoStdOutput:
        #print("--- HANDLER ----")
        root = logging.getLogger()
        handler = logging.StreamHandler(sys.stdout)
        formatter = logging.Formatter(format_str)
        handler.setFormatter(formatter)
        handler.setLevel(lev)
        root.addHandler(handler)
def getPathByRE(subpath, dir_re):
        pattern_cmp = re.compile(dir_re)
        lst = []
        if os.path.isdir(subpath):
            for fn in os.listdir(subpath):
                src_path =  os.path.join(subpath, fn)
                if os.path.isdir(src_path):
                    if pattern_cmp.match(fn):
                        lst += [src_path]
        return lst
def getPathByName(subpath, name):
        if os.path.isdir(subpath):
            for fn in os.listdir(subpath):
                src_path =  os.path.join(subpath, fn)
                if name == fn:
                    return fn
        return None
def getFirstBranchedDir(subpath):
    current_child=None
    if os.path.isdir(subpath):
        for fn in os.listdir(subpath):
            src_path =  os.path.join(subpath, fn)
            if (not fn.startswith(".")) and (os.path.isdir(src_path)):
                if current_child != None:
                    return subpath
                else:
                    current_child = fn
        if current_child != None:
            return getFirstBranchedDir(os.path.join(subpath, current_child))
    return subpath
def extractArchive(arch_name, dst_folder):
     with ZipFile(arch_name, 'r') as zipObj:
        zipObj.extractall(dst_folder)
def filename_extract(path):
    head, tail = os.path.split(path)
    return tail or os.path.basename(head)
if __name__ == "__main__":
    MY_VERSION = "0.1b1"
    CFG_FILENAME= "trdnt_idf_cfg.json"
    def dummyFunc(args):
        print("*** Trident IoT. Z-Way ESP Builder Platform:%s Version:%s ***"%(platform.system(), MY_VERSION))
        sys.exit(0) 
    def selectAlternative(lst, msg_select, msg_error, bQuietly=False):
        if len(lst) == 0:
            logging.error(msg_error)
            return None
        if (len(lst) == 1) or bQuietly:
            return lst[0]
        print(msg_select)
        i =1 
        for l in lst:
            print("%d-%s"%(i, l))
            i += 1
        while 1:
            ti = input("Which one do you need? [1-%d]:"%(len(lst)))
            ti = int(ti)
            if ti >= 1 or ti<=len(lst):
                return lst[ti-1]
            else:
                print("Wrong input. Select value in range[1-%d]"%(len(lst)))
        return None
    def checkPythonPackages(python_executable, requred_pkg_list, bInstall=True):
        logging.info("Python executable:%s"%(python_executable))
        python_args = [python_executable, "-m", "pip", "freeze"]
        ret_code, text = executeExternalSubprocess(python_args, "./", full_output=False)
        if ret_code != 0:
            logging.error("Unable to call pip! Error code:%d. Output:%s"%(ret_code, text))
            return False, []
        lines = text.split("\n")
        installed_pkgs = []
        for l in lines:
            pkg_data = l.split("==")
            if len(pkg_data) > 1:
                installed_pkgs += [pkg_data[0]]
        need_to_install = []
        text_to_install = ""
        for rp in requred_pkg_list:
            if not rp in installed_pkgs:
                need_to_install += [rp]
        if len(need_to_install) == 0:
            return True, []
        if not bInstall:
            return len(need_to_install) == 0, need_to_install
        python_args = [python_executable, "-m", "pip", "install"] + need_to_install
        ret_code, text = executeExternalSubprocess(python_args, "./", full_output=False)
        if ret_code != 0:
            logging.error("Unable to install packages via pip! Error code:%d. Output:%s"%(ret_code, text))
            return False, need_to_install
        return True, need_to_install
       
    def createEnv(idf_config, required_packages= [], ext_cfg=None, bQuietly=False):
        env = []
        idf_python_env = idf_config["python"]
        #idf_python = idf_config["python"]
        idf_tools = idf_config["tools"]
        idf_main = idf_config["idf"]
        
        env = os.environ.copy() 
        

        env["IDF_PYTHON_ENV_PATH"] = idf_python_env
        env["IDF_TOOLS_PATH"] = idf_tools
        env["IDF_PATH"] = idf_main
        
        current_path = "" 
        parttable_tool =  os.path.join(idf_main, "components", "partition_table")
        logging.info("parttable tool:%s"%(parttable_tool))
        if "elf_tools" not in idf_config:
            elf_tools_start_path = os.path.join(idf_tools, "tools", "xtensa-esp-elf")
            elf_tools_start = getPathByRE(elf_tools_start_path, "^esp-[0-9._]+$")

            elf_tools_v = selectAlternative(elf_tools_start, 
                          "We have different candidates for ELF tools:", 
                          "Unable to find ELF tools in idf directory:\"%s\""%(elf_tools_start_path))
            if elf_tools_v == None:
                return None
            
        else:
            elf_tools_v = idf_config["elf_tools"]
        elf_tools =   os.path.join(elf_tools_v, "xtensa-esp-elf", "bin")
        logging.info("ELF tools found:%s"%(elf_tools))
        # RISCV support
        if not "elf_tools_riscv" in idf_config:
            elf_tools_riscv_start_path = os.path.join(idf_tools, "tools", "riscv32-esp-elf")
            elf_tools_riscv_start = getPathByRE(elf_tools_riscv_start_path, "^esp-[0-9._]+$")

            elf_tools_riscv_v = selectAlternative(elf_tools_riscv_start, 
                                "We have different candidates for RISCV ELF tools:", 
                                "Unable to find ELF tools in idf directory:\"%s\""%(elf_tools_start_path))    
            if(elf_tools_riscv_v == None):
                return None
        else:
            elf_tools_riscv_v = idf_config["elf_tools_riscv"]
        elf_tools_riscv = os.path.join(elf_tools_riscv_v, "riscv32-esp-elf", "bin")
        logging.info("ELF RISCV tools found:%s"%(elf_tools))
        #elf_tools =   "%s"%(Path(idf_tools).joinpath("tools/xtensa-esp-elf/esp-13.2.0_20240530/xtensa-esp-elf/bin"))
        bin_dir = "bin"
        bin_ext = ""
        if sys.platform == 'win32':
            bin_dir = "Scripts" 
            bin_ext = ".exe" 
        python_env =   os.path.join(idf_python_env, bin_dir)#"%s"%(Path(idf_python_env).joinpath("bin"))
        python_executable3 = getPathByName(python_env, "python3"+bin_ext)
        python_executable = getPathByName(python_env, "python"+bin_ext)
        if python_executable3 != None:
            ext_cfg["python"] = os.path.join(python_env, python_executable3)
        elif python_executable != None:
            ext_cfg["python"] = os.path.join(python_env, python_executable)
        else:
            logging.error("Unable to locate python executable in:%s"%(python_env))
            return None
        if len(required_packages) != 0:
            logging.info("Checking for required python's packages:%s"%(required_packages))
            ready, pkg = checkPythonPackages(ext_cfg["python"], required_packages, True)
            if not ready:
                return None
        ext_cfg["python_env"] = python_env
        logging.info("ELF python executable found:%s"%(elf_tools))
        if not "elf_tools_roms" in idf_config:
            elf_roms_start_path = os.path.join(idf_tools, "tools", "esp-rom-elfs")
            elf_roms_start = getPathByRE(elf_roms_start_path, "^[0-9]+$")

            elf_roms_start_v = selectAlternative(elf_roms_start,
                                            "We have different candidates for ELF ROMS",
                                            "Unable to find ELF ROMs in idf directory:\"%s\""%(elf_roms_start_path)
                                            )
            if elf_roms_start_v == None:
                return None
        else:
            elf_roms_start_v = idf_config["elf_tools_roms"]
        roms = elf_roms_start_v
        logging.info("ELF ROMS found:%s"%(roms))
        cmake_start_path = os.path.join(idf_tools, "tools", "cmake")
        cmake_start = getPathByRE(cmake_start_path, "^[0-9\.]+$")
        cmake_start_v = selectAlternative(cmake_start,
                                            "We have different candidates for CMake",
                                            "Unable to find CMake in idf tools directory:\"%s\""%(cmake_start_path)
                                            )
        
        ninja_start_path = os.path.join(idf_tools, "tools", "ninja")
        ninja_start = getPathByRE(ninja_start_path, "^[0-9\.]+$")
        ninja_start_v = selectAlternative(ninja_start,
                                            "We have different candidates for Ninja",
                                            "Unable to find Ninja in idf tools directory:\"%s\""%(ninja_start_path)
                                            )
        
        sep = ":"
        if sys.platform == 'win32':
            sep = ";"
            
        current_path += parttable_tool
        current_path += sep + elf_tools
        current_path += sep + elf_tools_riscv
        current_path += sep + python_env
        current_path += sep + roms
        if cmake_start_v != None:
            if sys.platform == 'darwin':
                cmake_path = os.path.join(cmake_start_v, "CMake.app", "Contents", "bin")
            else:
                cmake_path = os.path.join(cmake_start_v, "bin")
            logging.info("CMake found:%s"%(cmake_path))
            current_path += sep + cmake_path
            if ext_cfg != None:
                ext_cfg["cmake"] = cmake_path

        if ninja_start_v != None:
            logging.info("Ninja found:%s"%(ninja_start_v))
            current_path += sep + ninja_start_v
            if ext_cfg != None:
                ext_cfg["ninja"] = ninja_start_v
        
        if sys.platform == 'win32':
            current_path = current_path.replace("/","\\")
        current_path += sep + os.environ["PATH"]
        env["PATH"] = current_path
        logging.info("  Current PATH:%s"%(current_path))
        logging.debug("Environment list:%s"%(env))
        return env
    def packArchCbk(current, total):
        sys.stdout.write("\r Packing archive: %d/%d "%(current, total))
        if(current == total):
            sys.stdout.write("\r Packing archive:[DONE]\n")
    def createMDDescriptions(filename, esp_target, main_name):
        boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32
        part_table_addr = ESP_ADDRESS_SPACE_PART_TABLE
        if(esp_target == "esp32s3"):
            boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32S3
        elif(esp_target == "esp32p4"):
            boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32P4
            part_table_addr = ESP_ADDRESS_SPACE_PART_TABLE_P4
        md_data={   "device_name":main_name,
                    "chip_type":esp_target, 
                    "build_ts":time.time(),
                    "file_data":{BOOTLOADER_FILENAME:{"address":boot_addr},
                                 PARTTABLE_FILENAME: {"address": part_table_addr},
                                 "%s.bin"%(main_name):{"address":ESP_ADDRESS_SPACE_MAIN}
                      }
                };
        dumpJSONData(filename, md_data)
    def buildDevice(idf_config, env_lst, source_dir, build_dir, device, esp_target, output_dir, main_name, ext_paths, add_ts=False, bRelease=False, test_lib=False, install_path=None, bNoClean=True):
        if (not bNoClean) and  os.path.isdir(build_dir):
            logging.info("--- Cleaning build directory %s ---"%(build_dir))
            shutil.rmtree(build_dir)
            if os.path.isdir(build_dir):
                print("!RMTREE failes")
                return -20
        if not os.path.isdir(build_dir):
            Path(build_dir).mkdir(parents=True, exist_ok=False)
        # IDF_TARGET=esp32p4
        sdk_config_file = os.path.join(source_dir, "sdkconfig")#Path(source_dir).joinpath("sdkconfig")
        
        if os.path.isfile(sdk_config_file) and (not bNoClean):
            logging.info("--- Removing sdkconfig %s ---"%(sdk_config_file))
            os.remove(str(sdk_config_file))
        build_dir_binarch = os.path.join(build_dir, "bin_arch")  #Path(build_dir).joinpath("bin_arch")
        if "cmake" in ext_paths:
            if ext_paths["cmake"].startswith("@"):
                cmake_executable = ext_paths["cmake"][1:]
            else:
                cmake_executable =  os.path.join(ext_paths["cmake"], "cmake")
        else:    
            cmake_executable  = shutil.which("cmake")
            if(cmake_executable == None):
                logging.error("Unable to find cmake in your system")
                return -11
        # Select a build type
        build_type = "Debug"
        if(bRelease):
            build_type = "Release"
        env_lst["IDF_TARGET"]=esp_target
        #env_lst["ESP_ROM_ELF_DIR"]="/Users/alexanderpolyakov/.espressif/tools/esp-rom-elfs/20240305/"
        start_v_pat = "esp"+os.sep+"v"
        start_vi = env_lst["IDF_PATH"].find(start_v_pat)
        end_vi = env_lst["IDF_PATH"].find(os.sep+"esp-idf")
        #logging.info("marks:%d %d "%( start_v, end_v))
        if (start_vi == -1) or (end_vi == -1):
            env_lst["ESP_IDF_VERSION"]="5.3"
        else:
            start_vi += len(start_v_pat)
            version_text = env_lst["IDF_PATH"][start_vi:end_vi]
            last_simb = version_text.rfind(".")
            env_lst["ESP_IDF_VERSION"]=version_text[:last_simb]
        logging.info("ESP_IDF_VERSION:%s"%( env_lst["ESP_IDF_VERSION"]))
        cmake_args = [cmake_executable, 
                      "-S",  source_dir, 
                      "-B", build_dir, 
                      "-D", "DEVICE=%s"%(device), 
                      "-D", "CMAKE_BUILD_TYPE=%s"%(build_type), 
                      #"-D", "IDF_TARGET=%s"%(esp_target), 
                      "-G", "Ninja", 
                      "-D", "PYTHON_DEPS_CHECKED=1",
                      "-D", "ESP_PLATFORM=1",  
                      "-D", "SDKCONFIG='%s'"%(sdk_config_file)
        ]
        # Add install prefix,
        if install_path != None and len(install_path) != 0:
            cmake_args += ["-D", "CMAKE_INSTALL_PREFIX=%s"%(install_path)]
        logging.info("CMAKE args:%s"%(cmake_args))
        ret_code, text = executeExternalSubprocess(cmake_args, "./",  env_list=env_lst, full_output=True, no_copy_env=True)
        if ret_code != 0:
            logging.error("CMAKE fails:%d", ret_code)
            return -3
        if "ninja" in ext_paths:
            ninja_executable =  os.path.join(ext_paths["ninja"], "ninja")
        else:
            ninja_executable  = shutil.which("ninja")
            if(ninja_executable == None):
                logging.error("Unable to find ninja on your system")
                return -12
        ninja_args = [ninja_executable]
        ret_code, text = executeExternalSubprocess(ninja_args, build_dir, env_list=env_lst, full_output=True, no_copy_env=True)
        if ret_code != 0:
            return -4
        # run install command if it is needed
        if install_path != None and len(install_path) != 0:
            ninja_install_args = [ninja_executable, "install"]
            ret_code, text = executeExternalSubprocess(ninja_install_args, build_dir, env_list=env_lst, full_output=True, no_copy_env=True)
            if ret_code != 0:
                return -5
            if test_lib:
                tmp_dir = tempfile.TemporaryDirectory()
                tmp_dir = os.path.join(tmp_dir.name, "srcs")
                shutil.copytree(install_path, tmp_dir)
                tmp_cmake_path = os.path.join(tmp_dir, "z-way-esp32")
                tmp_build_dir = os.path.join(tmp_cmake_path, "build")
                tmp_cmake_args = [cmake_executable, "-S", tmp_cmake_path, "-B", tmp_build_dir, "-D", "DEVICE=%s"%(device), "-D", "CMAKE_BUILD_TYPE=%s"%(build_type), "-D", "IDF_TARGET=%s"%(esp_target), "-G", "Ninja" ]
                ret_code, text = executeExternalSubprocess(tmp_cmake_args, tmp_cmake_path,  env_list=env_lst, full_output=True, no_copy_env=True)
                if ret_code != 0:
                    logging.error("CMAKE fails:%d", ret_code)
                    return -6
                ret_code, text = executeExternalSubprocess([ninja_executable], tmp_build_dir, env_list=env_lst, full_output=True, no_copy_env=True)
                if ret_code != 0:
                    logging.error("Ninja fails:%d", ret_code)
                    return -7
        # Copy all needed files to temporary directory
        bootloader_bin = os.path.join(build_dir, "bootloader/bootloader.bin") #Path(build_dir).joinpath("bootloader/bootloader.bin")
        partition_bin =  os.path.join(build_dir, "partition_table/partition-table.bin")#Path(build_dir).joinpath("partition_table/partition-table.bin")
        main_bin = os.path.join(build_dir, main_name + ".bin")
        Path(build_dir_binarch).mkdir(parents=True, exist_ok=True)
        logging.info("Created a temorary folder:%s"%(build_dir_binarch))
        shutil.copyfile(bootloader_bin, os.path.join(build_dir_binarch, BOOTLOADER_FILENAME))
        shutil.copyfile(partition_bin, os.path.join(build_dir_binarch, PARTTABLE_FILENAME))
        shutil.copyfile(main_bin, os.path.join(build_dir_binarch, main_name + ".bin"))
        createMDDescriptions(os.path.join(build_dir_binarch, MD_FILENAME), esp_target, main_name)
        if not os.path.isdir(output_dir):
            Path(output_dir).mkdir(parents=True, exist_ok=False)
        if add_ts:
            output_zip = os.path.join(output_dir, "%s-%s-%d.zip"%(main_name, device, int(time.time()))) # Path(output_dir).joinpath("%s-%s-%d.zip"%(main_name, device, int(time.time())))
        else:
            output_zip = os.path.join(output_dir, "%s-%s.zip"%(main_name, device)) # Path(output_dir).joinpath("%s-%s.zip"%(main_name, device))
        packArchive(build_dir_binarch, str(output_zip), progress_cbk=packArchCbk, fix_acess_map=None)
        return 0
    def saveCFG(md, local_cfg, global_cfg, bQuietly):
        if bQuietly:
            return
        save_prompt =  "Do you want to save configuration (N = no, L=locally, G=globally)[L]:"
        save_action = input(save_prompt)
        if(save_action == "L" or save_action == "" ):
            logging.info("Z-Way ESP configuration was stored:%s"%(local_cfg))
            dumpJSONData(local_cfg, md)
        elif  (save_action == "G"):
            logging.info("Z-Way ESP configuration was stored:%s"%(global_cfg))
            dumpJSONData(global_cfg, md)
    def extractIDFConfiguration(source_dir, esp_home=None, bQuietly=False):
        idf_cfg_filename = CFG_FILENAME
        local_cfg =   os.path.join(source_dir, idf_cfg_filename)#"%s"%(Path(source_dir).joinpath(idf_cfg_filename))
        logging.info("Local cfg %s.", local_cfg)
        if Path(local_cfg).exists():
            md  = loadJSONData(local_cfg)
            if md != None:
                return md
        current_user_dir = Path.home()
        if esp_home != None:
            current_user_dir = esp_home
        logging.info("We use user directory:%s"%(current_user_dir))
        global_cfg = os.path.join(current_user_dir, idf_cfg_filename) #"%s"%(Path(current_user_dir).joinpath(idf_cfg_filename))
        logging.info("Global cfg %s.", global_cfg)
        if Path(global_cfg).exists():
            md  = loadJSONData(global_cfg)
            if md != None:
                return md
        logging.info("Can't find IDF configuration file %s. Trying to create auto-config.", idf_cfg_filename)
        espressif_dir = os.path.join(current_user_dir, ".espressif") #"%s"%(Path.joinpath(current_user_dir, ".espressif"))
        esp_dir = os.path.join(current_user_dir, "esp")#"%s"%(Path.joinpath(current_user_dir, "esp"))
        logging.info("user:%s auto-cfg:espressif dir:\"%s\" esp_dir\"%s\""%(current_user_dir, espressif_dir, esp_dir))
        if(Path(espressif_dir).exists() and Path(esp_dir).exists()):
            python_start_path = os.path.join(espressif_dir, "python_env")
            python_path_list = getPathByRE(python_start_path, "^idf[0-9\.]+_py[0-9\.]+_env$")
            python_path_v = selectAlternative(python_path_list,
                                            "We have different candidates for python env",
                                            "Unable to find python environment in \"%s\""%(python_start_path)
                                            )
            if python_path_v != None:
                python_path = python_path_v
                logging.info("Found python environment:%s"%(python_path))
                tools_path = espressif_dir
                # try the CLI installation pattern (without version) first
                cli_idf_path = os.path.join(esp_dir, "esp-idf")
                idf_path = None
                if os.path.isdir(cli_idf_path):
                    if os.path.isdir(os.path.join(cli_idf_path, "tools")):
                        idf_path = cli_idf_path
                if idf_path == None:
                    idf_path_list = getPathByRE(esp_dir, "^v[0-9\.]+$")
                    idf_path_v = selectAlternative(idf_path_list,
                                                "We have different candidates for idf:",
                                               "Unable to find idf installation in \"%s\""%(esp_dir)
                                                )
                    if idf_path_v != None:
                        idf_path = os.path.join(idf_path_v, "esp-idf")
                if idf_path != None:
                    logging.info("Found IDF istallation:%s"%(idf_path))
                    md = {"python":python_path, "tools":tools_path, "idf":idf_path}
                    
                    saveCFG(md, local_cfg, global_cfg, bQuietly)
                    return md
        if bQuietly:
            logging.error("Quietly mode selected. There is only one way to setup IDF configuration in this mode. Please provide the file:\"%s\""%(global_cfg))
            return None
        logging.info("Unable to locate ESP istallation automatically. Please locate it yourself")
        python_path_prompt =  "IDF_PYTHON_ENV_PATH (like %s):"%("~/.espressif/python_env/idf5.3_py3.8_env")
        python_path = input(python_path_prompt)
        tools_path_prompt =  "IDF_TOOLS_PATH (like %s):"%("~/.espressif")
        tools_path = input(tools_path_prompt)
        idf_path_prompt =  "IDF_PATH (like %s):"%("~/esp/v5.3.1/esp-idf")
        idf_path = input(idf_path_prompt)
        md = {"python":python_path, "tools":tools_path, "idf":idf_path}
        saveCFG(md, local_cfg, global_cfg, bQuietly)
        return md
        
    
    def buildFunc(args):
        targ_list_filename = args.targets_metadata

        idf_conf = extractIDFConfiguration(args.source_dir, args.esp_home_folder, args.quietly)
        if idf_conf == None:
            logging.error("Unable to find IDF installation!")
            return -1000
        if args.targets_metadata == None:
            targ_list_filename = os.path.join(args.source_dir, "zway_targets.json")

        logging.info("--- Loading target list %s --- "%(targ_list_filename))
        md_targets = loadJSONData(targ_list_filename)
        if(md_targets == None):
            logging.info("Unable to load targets metadata:%s"%(args.targets_metadata))
            return -1
        ext_paths = {}
        env_lst = createEnv(idf_conf, md_targets["required_python_packages"], ext_paths, args.quietly)
        if env_lst == None:
            logging.error("Unable to create IDF environment!")
            return -1001
        if args.cmake_custom_path != None:
            ext_paths["cmake"] = "@%s"%(args.cmake_custom_path)
        bMakeExample = (args.pack_example != None) and (len(args.pack_example) != 0)
        example_install_path = None
        if bMakeExample:
            example_install_path = os.path.join(args.output_dir, args.pack_example)
            if not os.path.isdir(example_install_path):
                Path(example_install_path).mkdir(parents=True, exist_ok=False)
            example_archive = os.path.join(args.output_dir, args.pack_example + ".zip")
        bRelFlg = bMakeExample or args.release
        ret = 0 
        if args.device == None:
            index = 0
            # Build for the all devices
            for device in md_targets["devices"]:
               target = md_targets["devices"][device]["target"]
               main_name = md_targets["project_name"]
               logging.info(" Building firmware \"%s\"  for device  %s (%s)", main_name, device, target)
               ret = buildDevice(idf_conf, env_lst, args.source_dir, args.build_dir, device, target, args.output_dir, main_name, ext_paths, bRelease=bRelFlg, test_lib=args.test_lib, install_path=example_install_path, bNoClean=False)
               if ret == 0:
                   logging.info(" [DONE]")
               else:
                   logging.info(" [FAILED] error_code:%d", ret)
                   return -100-index
               index += 1
            #return 0
        else:
            build_md_filename = os.path.join(args.build_dir, MD_BUILD_FILENAME) 
            device = args.device
            if not device in md_targets["devices"]:
                logging.error("Unknown device:%s. Avaliable devices:[]", device)
                return -5
            target = md_targets["devices"][device]["target"]
            main_name = md_targets["project_name"]
            b_clean_rebuild = False
            build_md = {"device":device, "target":target, "main_name":main_name}
            # Load previos build metadata if we have one
            if os.path.isdir(args.build_dir):
                if os.path.isfile(build_md_filename):
                    build_md = loadJSONData(build_md_filename)
            # Check if we have to rebuild all
            b_clean_rebuild = (args.clean) or (build_md["device"] != device) or (build_md["target"] != target) or (build_md["main_name"] != main_name)
            build_md["ts"] = time.time()
            if b_clean_rebuild:
                logging.info("--- Clean rebuild. Settings have been changed! (dev:%s/%s target:%s/%s name: %s/%s) ----"%(build_md["device"], device, build_md["target"], target, build_md["main_name"], main_name))                                                                                
            ret = buildDevice(idf_conf, env_lst, args.source_dir, args.build_dir, device, target, args.output_dir, main_name, ext_paths, bRelease=bRelFlg, test_lib=args.test_lib,install_path=example_install_path, bNoClean=not b_clean_rebuild)
            if ret == 0:
                # Store current build settings
                dumpJSONData(build_md_filename, build_md)
        if bMakeExample:
            # pack an example code
            logging.info("--- Pack sources and libraries into %s ---"%(example_archive))
            packArchive(example_install_path, example_archive, progress_cbk=packArchCbk, fix_acess_map=None)
            logging.info("--- Cleaning example directory %s ---"%(example_install_path))
            shutil.rmtree(example_install_path)
        return ret
    def buildFuncExit(args):
        sys.exit(buildFunc(args))
    def diffFileNameInDir(dir, lst, ext):
        ol = []
        for fn in os.listdir(dir):
            print("FN:%s"%(fn))
            if os.path.isfile(os.path.join(dir, fn)) and fn.endswith(ext) and (not (fn in lst)):
                ol += [fn]
        return ol
    def flashFunc(args):
        current_directory = os.getcwd()
        home_directory =  Path.home()
        idf_conf = extractIDFConfiguration(current_directory, args.esp_home_folder, args.quietly)
        if idf_conf == None:
            logging.error("Unable to find IDF installation!")
            return -1000
        ext_paths = {}
        env_lst = createEnv(idf_conf, [], ext_paths, args.quietly)
        if env_lst == None:
            logging.error("Unable to create IDF environment!")
            return -1001
        if not os.path.isfile(args.file):
            logging.error("Unknown firmware archive:%s!"%(args.file))
            return -1
        file_name = filename_extract(args.file)
        temp_arch_extract_path = os.path.join(home_directory, file_name.replace(".zip",""))
        logging.info("Extracting archive:%s... to %s"%(args.file, temp_arch_extract_path))
        try:
            extractArchive(args.file, temp_arch_extract_path)
        except Exception as e:
            logging.error("Unable to extract archive:%s to %s. Exception:%s"%(args.file, temp_arch_extract_path, e))
            return -3
        # Old version (3.8) of python adds an extra parent directory to the archive 
        upper_path = getFirstBranchedDir(temp_arch_extract_path)
        logging.info("Archive path:%s"%(upper_path))
        bootloader_name = os.path.join(upper_path, BOOTLOADER_FILENAME)
        if not os.path.isfile(bootloader_name):
            logging.error("Unable to find bootloader executable in arcive:%s."%(args.file))
            return -4
        parttable_name = os.path.join(upper_path, PARTTABLE_FILENAME)
        if not os.path.isfile(parttable_name):
            logging.error("Unable to find partition table  in arcive:%s."%(args.file))
            return -4
        dfns = diffFileNameInDir(upper_path, [BOOTLOADER_FILENAME, PARTTABLE_FILENAME], ".bin")
        if len(dfns) != 1:
            logging.error("Unable to find main executable in arcive:%s. Variants:%s"%(args.file, dfns))
            return 
        main_name = os.path.join(upper_path, dfns[0])
        md_name =  os.path.join(upper_path, MD_FILENAME)
        boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32
        part_table_addr = ESP_ADDRESS_SPACE_PART_TABLE
        if args.chip_type == "esp32s3":
            boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32S3
        elif(args.chip_type == "esp32p4"):
            boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32P4
            part_table_addr = ESP_ADDRESS_SPACE_PART_TABLE_P4
        chip_type = args.chip_type
       
        main_addr = ESP_ADDRESS_SPACE_MAIN

        if os.path.isfile(md_name):
            md = loadJSONData(md_name)
            chip_type = md["chip_type"]
            boot_addr = md["file_data"][BOOTLOADER_FILENAME]["address"]
            part_table_addr = md["file_data"][PARTTABLE_FILENAME]["address"]
            main_addr = md["file_data"][dfns[0]]["address"]
            logging.error("Using embedded metadata device:%s chip_type:%s boot_addr:%08x part_table_addr:%08x main_code_addr:%08x"%(md["device_name"], chip_type, boot_addr,  part_table_addr, main_addr))
       
        esptool_args = [ext_paths["python"], 
                      "-m",  "esptool", 
                      "--chip", chip_type, 
                      "-p", args.port, 
                      "-b", "460800", 
                      "--before", "default_reset", "--after", "hard_reset",
                      "write_flash", 
                      "0x%x"%(boot_addr),  bootloader_name,
                      "0x%x"%(part_table_addr), parttable_name,
                      "0x%x"%(main_addr), main_name,
                      ]
        if args.fix_memory:
            esptool_args += ["--flash_mode", "dio", "--flash_size", args.flash_size, "--flash_freq", args.spi_baudrate]
        ret_code, text = executeExternalSubprocess(esptool_args, ext_paths["python_env"], full_output=True)
        # clean the temporary directory anyway
        shutil.rmtree(temp_arch_extract_path)
        if ret_code != 0:
            logging.error("ESPTool fails:%d", ret_code)
            return -10
        logging.info("{DONE}")
        return 0
    def eraseFunc(args):
        current_directory = os.getcwd()
        home_directory =  Path.home()
        idf_conf = extractIDFConfiguration(current_directory, args.esp_home_folder, args.quietly)
        if idf_conf == None:
            logging.error("Unable to find IDF installation!")
            return -1000
        ext_paths = {}
        env_lst = createEnv(idf_conf, [], ext_paths, args.quietly)
        if env_lst == None:
            logging.error("Unable to create IDF environment!")
            return -1001
        esptool_args = [ext_paths["python"], 
                      "-m",  "esptool", 
                      "--chip", args.chip_type, 
                      "-p", args.port, 
                      "erase_flash"
                      ]
        ret_code, text = executeExternalSubprocess(esptool_args, ext_paths["python_env"], full_output=True)
        if ret_code != 0:
            logging.error("ESPTool fails:%d", ret_code)
            return -10
    def mergeFunc(args):
        current_directory = os.getcwd()
        home_directory =  Path.home()
        idf_conf = extractIDFConfiguration(current_directory, args.esp_home_folder, args.quietly)
        if idf_conf == None:
            logging.error("Unable to find IDF installation!")
            return -1000
        ext_paths = {}
        env_lst = createEnv(idf_conf, [], ext_paths, args.quietly)
        if env_lst == None:
            logging.error("Unable to create IDF environment!")
            return -1001
        if not os.path.isfile(args.file):
            logging.error("Unknown firmware archive:%s!"%(args.file))
            return -1
        file_name = filename_extract(args.file)
        temp_arch_extract_path = os.path.join(home_directory, file_name.replace(".zip",""))
        logging.info("Extracting archive:%s... to %s"%(args.file, temp_arch_extract_path))
        try:
            extractArchive(args.file, temp_arch_extract_path)
        except Exception as e:
            logging.error("Unable to extract archive:%s to %s. Exception:%s"%(args.file, temp_arch_extract_path, e))
            return -3
        # Old version (3.8) of python adds an extra parent directory to the archive 
        upper_path = getFirstBranchedDir(temp_arch_extract_path)
        logging.info("Archive path:%s"%(upper_path))
        bootloader_name = os.path.join(upper_path, BOOTLOADER_FILENAME)
        if not os.path.isfile(bootloader_name):
            logging.error("Unable to find bootloader executable in arcive:%s."%(args.file))
            return -4
        parttable_name = os.path.join(upper_path, PARTTABLE_FILENAME)
        if not os.path.isfile(parttable_name):
            logging.error("Unable to find partition table  in arcive:%s."%(args.file))
            return -4
        dfns = diffFileNameInDir(upper_path, [BOOTLOADER_FILENAME, PARTTABLE_FILENAME], ".bin")
        if len(dfns) != 1:
            logging.error("Unable to find main executable in arcive:%s. Variants:%s"%(args.file, dfns))
            return 
        main_name = os.path.join(upper_path, dfns[0])
        md_name =  os.path.join(upper_path, MD_FILENAME)
        boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32
        if args.chip_type == "esp32s3":
            boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32S3
        elif(args.chip_type == "esp32p4"):
            boot_addr = ESP_ADDRESS_SPACE_BOOT_ESP32P4
        chip_type = args.chip_type
        part_table_addr = ESP_ADDRESS_SPACE_PART_TABLE
        main_addr = ESP_ADDRESS_SPACE_MAIN

        if os.path.isfile(md_name):
            md = loadJSONData(md_name)
            chip_type = md["chip_type"]
            boot_addr = md["file_data"][BOOTLOADER_FILENAME]["address"]
            part_table_addr = md["file_data"][PARTTABLE_FILENAME]["address"]
            main_addr = md["file_data"][dfns[0]]["address"]
            logging.error("Using embedded metadata device:%s chip_type:%s boot_addr:%08x part_table_addr:%08x main_code_addr:%08x"%(md["device_name"], chip_type, boot_addr,  part_table_addr, main_addr))
       
        esptool_args = [ext_paths["python"], 
                      "-m",  "esptool", 
                      "--chip", chip_type, 
                      "merge_bin",
                      "-o", args.output_file,
                      "--flash_mode",  "dio",
                      "--flash_freq",  "40m",
                      "--flash_size",  args.memory_size,
                      "0x%x"%(boot_addr),  bootloader_name,
                      "0x%x"%(part_table_addr), parttable_name,
                      "0x%x"%(main_addr), main_name
                      ]
        ret_code, text = executeExternalSubprocess(esptool_args, ext_paths["python_env"], full_output=True)
        # clean the temporary directory anyway
        shutil.rmtree(temp_arch_extract_path)
        if ret_code != 0:
            logging.error("ESPTool fails:%d", ret_code)
            return -10
        logging.info("{DONE}")
    def flashFuncExit(args):
        sys.exit(flashFunc(args))
    def eraseFuncExit(args):
        sys.exit(eraseFunc(args))
    def mergeFuncExit(args):
        sys.exit(mergeFunc(args))
    def monitorFunc(args):
        current_directory = os.getcwd()
        idf_conf = extractIDFConfiguration(current_directory, args.esp_home_folder, args.quietly)
        if idf_conf == None:
            logging.error("Unable to find IDF installation!")
            return -1000
        ext_paths = {}
        env_lst = createEnv(idf_conf, [], ext_paths, args.quietly)
        if env_lst == None:
            logging.error("Unable to create IDF environment!")
            return -1001
        idf_monitor_script = os.path.join(idf_conf["idf"], "tools", "idf_monitor.py")
        esptool_args = [ext_paths["python"], 
                       idf_monitor_script, 
                      "-p", args.port, 
                      "-b", args.baud_rate
                      ]
        ret_code, text = executeExternalSubprocess(esptool_args, ext_paths["python_env"], full_output=True)
        return ret_code
    def monitorFuncExit(args):
        sys.exit(monitorFunc(args))
    def Main():
        zmeSetupLogging("ZWayESPBuilder", True, False)
        logging.debug("\nStarting on %s.\nARGS:%s\nVERSION:%s MD5:%s" % (
            platform.system(), ' '.join(sys.argv), MY_VERSION, "-"))
        parser = argparse.ArgumentParser(description='Z-Way Builder tool.\n Version:%s\n Welcome :) '%(MY_VERSION))
        parser.set_defaults(func=dummyFunc)
        subparsers = parser.add_subparsers()
        parserBuild = subparsers.add_parser('build', help="Builds ESP32 targets")
        parserBuild.add_argument('-b', '--build_dir', default="tmp_build",help="Build directory")
        parserBuild.add_argument('-s', '--source_dir', default="./",help="Source directory")
        parserBuild.add_argument('-o', '--output_dir', default="./bin",help="Output directory for binary archive")
        parserBuild.add_argument('-tf', '--targets_metadata', default=None,help="JSON file that contains all the targets and devices we use in this project")
        parserBuild.add_argument('-q', '--quietly', nargs='?', type=bool, const=True, default=False, help="Do not ask anything")
        parserBuild.add_argument('-r', '--release', nargs='?', type=bool, const=True, default=False, help="Make a release build.")
        parserBuild.add_argument('-0', '--clean', nargs='?', type=bool, const=True, default=False, help="Force clean the build directory")
        parserBuild.add_argument('-cm', '--cmake_custom_path', default=None, help="Define alternative cmake executable")
        parserBuild.add_argument('-pe', '--pack_example', default=None, help="Pack all the libraries and example code into archive. User is able to define package name.")
        parserBuild.add_argument('-d', '--device', default=None,help="Select the needed target device. If it's empty, all targets will build. Look at zway_targets.json for full list of supported targets.")
        parserBuild.add_argument('-eh', '--esp_home_folder', default=None,help="Sets ESP home folder.")
        parserBuild.add_argument('-tst', '--test_lib', type=bool, default=False, help="Testing the build from lib for errors")
        parserBuild.set_defaults(func=buildFuncExit)
        parserFlash = subparsers.add_parser('flash', help="Flashes zip-archive of firmware to ESP board")
        parserFlash.add_argument('file',help="zip-file with ESP firmware")
        parserFlash.add_argument('-p', '--port', help="Serial port/tty of ESP")
        parserFlash.add_argument('-c', '--chip_type', default="esp32", choices=SUPPORTED_CHIP_TYPES,help="ESP chip type")
        parserFlash.add_argument('-q', '--quietly', nargs='?', type=bool, const=True, default=False, help="Do not ask anything")
        parserFlash.add_argument('-f', '--fix_memory', nargs='?', type=bool, const=True, default=False, help="Add flash memory options. Autotedect size and etc.")
        parserFlash.add_argument('-s', '--spi_baudrate',  default="40m", help="SPI flash/PRAM speed. Is suitable only with -f flag.")
        parserFlash.add_argument('-z', '--flash_size',  default="detect", help="Modify default Flash memory size. Is suitable only with -f flag.")
        parserFlash.add_argument('-eh', '--esp_home_folder', default=None,help="Sets ESP home folder.")
        parserFlash.set_defaults(func=flashFuncExit)
        parserErase = subparsers.add_parser('erase', help="Erase all flash memory of ESP32 board")
        parserErase.add_argument('-p', '--port', help="Serial port/tty of ESP")
        parserErase.add_argument('-c', '--chip_type', default="esp32", choices=SUPPORTED_CHIP_TYPES, help="ESP chip type")
        parserErase.add_argument('-q', '--quietly', nargs='?', type=bool, const=True, default=False, help="Do not ask anything")
        parserErase.add_argument('-eh', '--esp_home_folder', default=None,help="Sets ESP home folder.")
        parserErase.set_defaults(func=eraseFuncExit)
        parserMerge = subparsers.add_parser('merge', help="Merges all the firmware data from zip file in one image")
        parserMerge.add_argument('file',help="zip-file with ESP firmware")
        parserMerge.add_argument('-o', '--output_file', default="a.bin", help="Output image")
        parserMerge.add_argument('-m', '--memory_size', default="4MB", help="Flash memory size of device")
        parserMerge.add_argument('-c', '--chip_type', default="esp32", choices=SUPPORTED_CHIP_TYPES,help="ESP chip type")
        parserMerge.add_argument('-q', '--quietly', nargs='?', type=bool, const=True, default=False, help="Do not ask anything")
        parserMerge.add_argument('-eh', '--esp_home_folder', default=None,help="Sets ESP home folder.")
        parserMerge.set_defaults(func=mergeFuncExit)
        parserMonitor = subparsers.add_parser('monitor', help="Starts monitor for selected ESP port. ")
        parserMonitor.add_argument('-p', '--port', help="Serial port/tty of ESP")
        parserMonitor.add_argument('-b', '--baud_rate', default="921600", choices=["921600", "460800", "115200"], help="Baudrate of system console")
        parserMonitor.add_argument('-eh', '--esp_home_folder', default=None,help="Sets ESP home folder.")
        parserMonitor.add_argument('-q', '--quietly', nargs='?', type=bool, const=True, default=False, help="Do not ask anything")
        parserMonitor.set_defaults(func=monitorFuncExit)
        args = parser.parse_args()
        args.func(args)
    Main()